/*
Copyright 2025 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2enode

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/onsi/ginkgo/v2"
	"github.com/onsi/gomega"
	"github.com/onsi/gomega/gcustom"
	"github.com/onsi/gomega/types"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/features"
	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
	admissionapi "k8s.io/pod-security-admission/api"
	"k8s.io/utils/cpuset"

	"k8s.io/kubernetes/test/e2e/feature"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
)

const (
	defaultCFSPeriod = "100000"
)

// this is ugly, but pratical
var (
	e2enodeRuntimeName     string
	e2enodeCgroupV2Enabled bool
	e2enodeCgroupDriver    string
)

/*
   - Serial:
   because the test updates kubelet configuration.

   - Ordered:
   Each spec (It block) need to run with a kubelet configuration in place. At minimum, we need
   the non-default cpumanager static policy, then we have the cpumanager options and so forth.
   The simplest solution is to set the kubelet explicitly each time, but this will cause a kubelet restart
   each time, which takes longer and makes the flow intrinsically more fragile (so more flakes are more likely).
   Using Ordered allows us to use BeforeAll/AfterAll, and most notably to reuse the kubelet config in a batch
   of specs (It blocks). Each it block will still set its kubelet config preconditions, but with a sensible
   test arrangement, many of these preconditions will devolve into noop.
   Arguably, this decision increases the coupling among specs, leaving room for subtle ordering bugs.
   There's no argue the ginkgo spec randomization would help, but the tradeoff here is between
   lane complexity/fragility (reconfiguring the kubelet is not bulletproof yet) and accepting this risk.
   If in the future we decide to pivot to make each spec fully independent, little changes will be needed.
   Finally, worth pointing out that the previous cpumanager e2e test incarnation implemented the same
   concept in a more convoluted way with function helpers, so arguably using Ordered and making it
   explicit is already an improvement.
*/

/*
 * Extending the cpumanager test suite
 * TL;DRs: THE MOST IMPORTANT:
 * Please keep the test hierarchy very flat.
 * Nesting more than 2 contexts total from SIGDescribe root is likely to be a smell.
 * The problem with deep nesting is the interaction between BeforeEach blocks and the increased scope of variables.
 * The Ideal layout would be
 *   SIGDescribe # top level, unique
 *     Context   # you can add more context, but please try hard to avoid nesting them
 *       It      # add it blocks freely. Feel free to start new *non-nested* contexts as you see fit
 *     Context
 *       It
 *       It
 * Exception: if you need to add the same labels to quite a few (say, 3+) It blocks, you can add a **empty** context
 * The most important thing is to avoid long chain of beforeeach/aftereach and > 2 level of context nesting.
 * So a **empty** context only to group labels is acceptable:
 *   SIGDescribe
 *     Context(label1, label2) # avoid beforeeach/aftereach and variables here
 *       Context
 *         It
 *         It
 *     Context
 *       It
 * Final rule of thumb: if the nesting of the context description starts to read awkward or funny or stop making sense
 * if read as english sentence, then the nesting is likely too deep.
 */

/*
 * About reserved CPUs:
 * all the tests assume the first available CPUID is 0, which is pretty fair and most of time correct.
 * A better approach would be check what we do have in the node. This is deferred to a later stage alongside
 * other improvements.
 */
var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, ginkgo.ContinueOnFailure, framework.WithSerial(), feature.CPUManager, func() {
	f := framework.NewDefaultFramework("cpumanager-test")
	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged

	// original kubeletconfig before the context start, to be restored
	var oldCfg *kubeletconfig.KubeletConfiguration
	var reservedCPUs cpuset.CPUSet
	var onlineCPUs cpuset.CPUSet
	var smtLevel int
	var uncoreGroupSize int
	// tracks all the pods created by a It() block. Best would be a namespace per It block
	// TODO: move to a namespace per It block?
	var podMap map[string]*v1.Pod

	// closure just and only to not carry around awkwardly `f` and `onlineCPUs` only for logging purposes
	var skipIfAllocatableCPUsLessThan func(node *v1.Node, cpuReq int)

	ginkgo.BeforeAll(func(ctx context.Context) {
		var err error
		oldCfg, err = getCurrentKubeletConfig(ctx)
		framework.ExpectNoError(err)

		onlineCPUs, err = getOnlineCPUs() // this should not change at all, at least during this suite lifetime
		framework.ExpectNoError(err)
		framework.Logf("Online CPUs: %s", onlineCPUs)

		smtLevel = smtLevelFromSysFS() // this should not change at all, at least during this suite lifetime
		framework.Logf("SMT level: %d", smtLevel)

		uncoreGroupSize = getUncoreCPUGroupSize()
		framework.Logf("Uncore Group Size: %d", uncoreGroupSize)

		e2enodeCgroupV2Enabled = IsCgroup2UnifiedMode()
		framework.Logf("cgroup V2 enabled: %v", e2enodeCgroupV2Enabled)

		e2enodeCgroupDriver = oldCfg.CgroupDriver
		framework.Logf("cgroup driver: %s", e2enodeCgroupDriver)

		runtime, _, err := getCRIClient()
		framework.ExpectNoError(err, "Failed to get CRI client")

		version, err := runtime.Version(context.Background(), "")
		framework.ExpectNoError(err, "Failed to get runtime version")

		e2enodeRuntimeName = version.GetRuntimeName()
		framework.Logf("runtime: %s", e2enodeRuntimeName)
	})

	ginkgo.AfterAll(func(ctx context.Context) {
		updateKubeletConfig(ctx, f, oldCfg, true)
	})

	ginkgo.BeforeEach(func(ctx context.Context) {
		// note intentionally NOT set reservedCPUs -  this must be initialized on a test-by-test basis
		podMap = make(map[string]*v1.Pod)
	})

	ginkgo.JustBeforeEach(func(ctx context.Context) {
		// note intentionally NOT set reservedCPUs -  this must be initialized on a test-by-test basis

		// use a closure to minimize the arguments, to make the usage more straightforward
		skipIfAllocatableCPUsLessThan = func(node *v1.Node, val int) {
			ginkgo.GinkgoHelper()
			cpuReq := int64(val + reservedCPUs.Size()) // reserved CPUs are not usable, need to account them
			// the framework is initialized using an injected BeforeEach node, so the
			// earliest we can do is to initialize the other objects here
			nodeCPUDetails := cpuDetailsFromNode(node)

			msg := fmt.Sprintf("%v full CPUs (detected=%v requested=%v reserved=%v online=%v smt=%v)", cpuReq, nodeCPUDetails.Allocatable, val, reservedCPUs.Size(), onlineCPUs.Size(), smtLevel)
			ginkgo.By("Checking if allocatable: " + msg)
			if nodeCPUDetails.Allocatable < cpuReq {
				e2eskipper.Skipf("Skipping CPU Manager test: not allocatable %s", msg)
			}
		}
	})

	ginkgo.AfterEach(func(ctx context.Context) {
		deletePodsAsync(ctx, f, podMap)
	})

	ginkgo.When("running non-guaranteed pods tests", ginkgo.Label("non-guaranteed", "reserved-cpus"), func() {
		ginkgo.It("should let the container access all the online CPUs without a reserved CPUs set", func(ctx context.Context) {
			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: cpuset.CPUSet{},
			}))

			pod := makeCPUManagerPod("non-gu-pod", []ctnAttribute{
				{
					ctnName:    "non-gu-container",
					cpuRequest: "100m",
					cpuLimit:   "200m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			gomega.Expect(pod).To(HaveContainerCPUsEqualTo("non-gu-container", onlineCPUs))
		})

		ginkgo.It("should let the container access all the online CPUs when using a reserved CPUs set", func(ctx context.Context) {
			reservedCPUs = cpuset.New(0)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("non-gu-pod", []ctnAttribute{
				{
					ctnName:    "non-gu-container",
					cpuRequest: "100m",
					cpuLimit:   "200m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			gomega.Expect(pod).To(HaveContainerCPUsEqualTo("non-gu-container", onlineCPUs))
		})

		ginkgo.It("should let the container access all the online non-exclusively-allocated CPUs when using a reserved CPUs set", ginkgo.Label("guaranteed", "exclusive-cpus"), func(ctx context.Context) {
			cpuCount := 1
			reservedCPUs = cpuset.New(0)

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount+1) // note the extra for the non-gu pod

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			podGu := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
					cpuLimit:   fmt.Sprintf("%dm", 1000*cpuCount),
				},
			})
			ginkgo.By("creating the guaranteed test pod")
			podGu = e2epod.NewPodClient(f).CreateSync(ctx, podGu)
			podMap[string(podGu.UID)] = podGu

			podBu := makeCPUManagerPod("non-gu-pod", []ctnAttribute{
				{
					ctnName:    "non-gu-container",
					cpuRequest: "200m",
					cpuLimit:   "300m",
				},
			})
			ginkgo.By("creating the burstable test pod")
			podBu = e2epod.NewPodClient(f).CreateSync(ctx, podBu)
			podMap[string(podBu.UID)] = podBu

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(podGu).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(podGu).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
			gomega.Expect(podGu).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))

			exclusiveCPUs, err := getContainerAllowedCPUs(podGu, "gu-container", false)
			framework.ExpectNoError(err, "cannot get exclusive CPUs for pod %s/%s", podGu.Namespace, podGu.Name)
			expectedSharedCPUs := onlineCPUs.Difference(exclusiveCPUs)
			gomega.Expect(podBu).To(HaveContainerCPUsEqualTo("non-gu-container", expectedSharedCPUs))
		})
	})

	ginkgo.When("running guaranteed pod tests", ginkgo.Label("guaranteed", "exclusive-cpus"), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {
			reservedCPUs = cpuset.New(0)
		})

		ginkgo.It("should allocate exclusively a CPU to a 1-container pod", func(ctx context.Context) {
			cpuCount := 1

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
					cpuLimit:   fmt.Sprintf("%dm", 1000*cpuCount),
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
		})

		// we don't use a separate group (gingo.When) with BeforeEach to factor out the tests because each
		// test need to check for the amount of CPUs it needs.

		ginkgo.It("should allocate exclusively a even number of CPUs to a 1-container pod", func(ctx context.Context) {
			cpuCount := 2

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
					cpuLimit:   fmt.Sprintf("%dm", 1000*cpuCount),
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
			// TODO: this is probably too strict but it is the closest of the old test did
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container"))
		})

		ginkgo.It("should allocate exclusively a odd number of CPUs to a 1-container pod", func(ctx context.Context) {
			cpuCount := 3

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
					cpuLimit:   fmt.Sprintf("%dm", 1000*cpuCount),
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
			// TODO: this is probably too strict but it is the closest of the old test did
			toleration := 1
			gomega.Expect(pod).To(HaveContainerCPUsQuasiThreadSiblings("gu-container", toleration))
		})

		ginkgo.It("should allocate exclusively CPUs to a multi-container pod (1+2)", func(ctx context.Context) {
			cpuCount := 3 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "1000m",
					cpuLimit:   "1000m",
				},
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 1))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))

			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			// TODO: this is probably too strict but it is the closest of the old test did
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})

		ginkgo.It("should allocate exclusively CPUs to a multi-container pod (3+2)", func(ctx context.Context) {
			cpuCount := 5 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "3000m",
					cpuLimit:   "3000m",
				},
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 3))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))
			toleration := 1
			gomega.Expect(pod).To(HaveContainerCPUsQuasiThreadSiblings("gu-container-1", toleration))

			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})

		ginkgo.It("should allocate exclusively CPUs to a multi-container pod (4+2)", func(ctx context.Context) {
			cpuCount := 6 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "4000m",
					cpuLimit:   "4000m",
				},
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 4))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-1"))

			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})

		ginkgo.It("should allocate exclusively a CPU to multiple 1-container pods", func(ctx context.Context) {
			cpuCount := 4 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod1 := makeCPUManagerPod("gu-pod-1", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod 1")
			pod1 = e2epod.NewPodClient(f).CreateSync(ctx, pod1)
			podMap[string(pod1.UID)] = pod1

			pod2 := makeCPUManagerPod("gu-pod-2", []ctnAttribute{
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod 2")
			pod2 = e2epod.NewPodClient(f).CreateSync(ctx, pod2)
			podMap[string(pod2.UID)] = pod2

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod1).To(HaveContainerCPUsCount("gu-container-1", 2))
			gomega.Expect(pod1).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod1).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))
			gomega.Expect(pod1).To(HaveContainerCPUsThreadSiblings("gu-container-1"))

			gomega.Expect(pod2).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod2).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod2).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			gomega.Expect(pod2).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})
	})

	ginkgo.When("running guaranteed pod tests with feature gates disabled", ginkgo.Label("guaranteed", "exclusive-cpus", "feature-gate-disabled"), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {
			reservedCPUs = cpuset.New(0)
		})

		ginkgo.It("should allocate exclusively a CPU to a 1-container pod", func(ctx context.Context) {
			cpuCount := 1

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: false,
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
					cpuLimit:   fmt.Sprintf("%dm", 1000*cpuCount),
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
		})

		// we don't use a separate group (gingo.When) with BeforeEach to factor out the tests because each
		// test need to check for the amount of CPUs it needs.

		ginkgo.It("should allocate exclusively a even number of CPUs to a 1-container pod", func(ctx context.Context) {
			cpuCount := 2

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: false,
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
					cpuLimit:   fmt.Sprintf("%dm", 1000*cpuCount),
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
			// TODO: this is probably too strict but it is the closest of the old test did
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container"))
		})

		ginkgo.It("should allocate exclusively a odd number of CPUs to a 1-container pod", func(ctx context.Context) {
			cpuCount := 3

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: false,
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
					cpuLimit:   fmt.Sprintf("%dm", 1000*cpuCount),
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
			// TODO: this is probably too strict but it is the closest of the old test did
			toleration := 1
			gomega.Expect(pod).To(HaveContainerCPUsQuasiThreadSiblings("gu-container", toleration))
		})

		ginkgo.It("should allocate exclusively CPUs to a multi-container pod (1+2)", func(ctx context.Context) {
			cpuCount := 3 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: false,
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "1000m",
					cpuLimit:   "1000m",
				},
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 1))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))

			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			// TODO: this is probably too strict but it is the closest of the old test did
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})

		ginkgo.It("should allocate exclusively CPUs to a multi-container pod (3+2)", func(ctx context.Context) {
			cpuCount := 5 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: false,
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "3000m",
					cpuLimit:   "3000m",
				},
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 3))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))
			toleration := 1
			gomega.Expect(pod).To(HaveContainerCPUsQuasiThreadSiblings("gu-container-1", toleration))

			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})

		ginkgo.It("should allocate exclusively CPUs to a multi-container pod (4+2)", func(ctx context.Context) {
			cpuCount := 6 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: false,
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "4000m",
					cpuLimit:   "4000m",
				},
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 4))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-1"))

			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})

		ginkgo.It("should allocate exclusively a CPU to multiple 1-container pods", func(ctx context.Context) {
			cpuCount := 4 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: false,
			}))

			pod1 := makeCPUManagerPod("gu-pod-1", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod 1")
			pod1 = e2epod.NewPodClient(f).CreateSync(ctx, pod1)
			podMap[string(pod1.UID)] = pod1

			pod2 := makeCPUManagerPod("gu-pod-2", []ctnAttribute{
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod 2")
			pod2 = e2epod.NewPodClient(f).CreateSync(ctx, pod2)
			podMap[string(pod2.UID)] = pod2

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod1).To(HaveContainerCPUsCount("gu-container-1", 2))
			gomega.Expect(pod1).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod1).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))
			gomega.Expect(pod1).To(HaveContainerCPUsThreadSiblings("gu-container-1"))

			gomega.Expect(pod2).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod2).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod2).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			gomega.Expect(pod2).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})
	})

	ginkgo.When("running with strict CPU reservation", ginkgo.Label("strict-cpu-reservation"), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {
			reservedCPUs = cpuset.New(0)
		})

		ginkgo.It("should let the container access all the online CPUs without a reserved CPUs set", func(ctx context.Context) {
			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      cpuset.CPUSet{},
				enableCPUManagerOptions: true,
				options: map[string]string{
					cpumanager.StrictCPUReservationOption: "true",
				},
			}))

			pod := makeCPUManagerPod("non-gu-pod", []ctnAttribute{
				{
					ctnName:    "non-gu-container",
					cpuRequest: "100m",
					cpuLimit:   "200m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// cpumanager will always reserve at least 1 cpu. In this case we don't set which, and if we treat the cpumanager
			// as black box (which we very much should) we can't predict which one. So we can only assert that *A* cpu is not
			// usable because is reserved.
			gomega.Expect(pod).To(HaveContainerCPUsCount("non-gu-container", onlineCPUs.Size()-1))
		})

		ginkgo.It("should let the container access all the online CPUs minus the reserved CPUs set when enabled", func(ctx context.Context) {
			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: true,
				options: map[string]string{
					cpumanager.StrictCPUReservationOption: "true",
				},
			}))

			pod := makeCPUManagerPod("non-gu-pod", []ctnAttribute{
				{
					ctnName:    "non-gu-container",
					cpuRequest: "100m",
					cpuLimit:   "200m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			gomega.Expect(pod).To(HaveContainerCPUsEqualTo("non-gu-container", onlineCPUs.Difference(reservedCPUs)))
		})

		ginkgo.It("should let the container access all the online non-exclusively-allocated CPUs minus the reserved CPUs set when enabled", func(ctx context.Context) {
			cpuCount := 1

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount+1) // note the extra for the non-gu pod)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enableCPUManagerOptions: true,
				options: map[string]string{
					cpumanager.StrictCPUReservationOption: "true",
				},
			}))

			cpuReq := fmt.Sprintf("%dm", 1000*cpuCount)

			podGu := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: cpuReq,
					cpuLimit:   cpuReq,
				},
			})
			ginkgo.By("creating the guaranteed test pod")
			podGu = e2epod.NewPodClient(f).CreateSync(ctx, podGu)
			podMap[string(podGu.UID)] = podGu

			podBu := makeCPUManagerPod("non-gu-pod", []ctnAttribute{
				{
					ctnName:    "non-gu-container",
					cpuRequest: "200m",
					cpuLimit:   "300m",
				},
			})
			ginkgo.By("creating the burstable test pod")
			podBu = e2epod.NewPodClient(f).CreateSync(ctx, podBu)
			podMap[string(podBu.UID)] = podBu

			ginkgo.By("checking if the expected cpuset was assigned")

			usableCPUs := onlineCPUs.Difference(reservedCPUs)

			// any full CPU is fine - we cannot nor we should predict which one, though
			gomega.Expect(podGu).To(HaveContainerCPUsCount("gu-container", cpuCount))
			gomega.Expect(podGu).To(HaveContainerCPUsASubsetOf("gu-container", usableCPUs))
			gomega.Expect(podGu).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))

			exclusiveCPUs, err := getContainerAllowedCPUs(podGu, "gu-container", false)
			framework.ExpectNoError(err, "cannot get exclusive CPUs for pod %s/%s", podGu.Namespace, podGu.Name)
			expectedSharedCPUs := usableCPUs.Difference(exclusiveCPUs)
			gomega.Expect(podBu).To(HaveContainerCPUsEqualTo("non-gu-container", expectedSharedCPUs))
		})
	})

	ginkgo.When("running with SMT Alignment", ginkgo.Label("smt-alignment"), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {
			// strict SMT alignment is trivially verified and granted on non-SMT systems
			if smtLevel < minSMTLevel {
				e2eskipper.Skipf("Skipping CPU Manager %q tests since SMT disabled", cpumanager.FullPCPUsOnlyOption)
			}

			reservedCPUs = cpuset.New(0)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs,
				enableCPUManagerOptions: true,
				options: map[string]string{
					cpumanager.FullPCPUsOnlyOption: "true",
				},
			}))
		})

		ginkgo.It("should reject workload asking non-SMT-multiple of cpus", func(ctx context.Context) {
			cpuCount := 1
			cpuReq := fmt.Sprintf("%dm", 1000*cpuCount)

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			ginkgo.By("creating the testing pod")
			// negative test: try to run a container whose requests aren't a multiple of SMT level, expect a rejection
			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-neg",
					cpuRequest: cpuReq,
					cpuLimit:   cpuReq,
				},
			})
			ginkgo.By("creating the test pod")
			// CreateSync would wait for pod to become Ready - which will never happen if production code works as intended!
			pod = e2epod.NewPodClient(f).Create(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("ensuring the testing pod is in failed state")
			err := e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "Failed", 30*time.Second, func(pod *v1.Pod) (bool, error) {
				if pod.Status.Phase != v1.PodPending {
					return true, nil
				}
				return false, nil
			})
			framework.ExpectNoError(err)

			ginkgo.By("ensuring the testing pod is failed for the expected reason")
			pod, err = e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
			framework.ExpectNoError(err)
			gomega.Expect(pod).To(BeAPodInPhase(v1.PodFailed))
			gomega.Expect(pod).To(HaveStatusReasonMatchingRegex(`SMT.*Alignment.*Error`))
		})

		ginkgo.It("should admit workload asking SMT-multiple of cpus", func(ctx context.Context) {
			// positive test: try to run a container whose requests are a multiple of SMT level, check allocated cores
			// 1. are core siblings
			// 2. take a full core
			// WARNING: this assumes 2-way SMT systems - we don't know how to access other SMT levels.
			//          this means on more-than-2-way SMT systems this test will prove nothing
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), smtLevel)

			cpuRequest := fmt.Sprintf("%d000m", smtLevel)
			ginkgo.By(fmt.Sprintf("creating the testing pod cpuRequest=%v", cpuRequest))
			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-pos",
					cpuRequest: cpuRequest,
					cpuLimit:   cpuRequest,
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("validating each container in the testing pod")
			for _, cnt := range pod.Spec.Containers {
				ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

				gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
				gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
			}
		})
	})

	ginkgo.When("running with Uncore Cache Alignment", ginkgo.Label("prefer-align-cpus-by-uncore-cache"), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {

			reservedCPUs := cpuset.New(0)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs,
				enableCPUManagerOptions: true,
				options: map[string]string{
					cpumanager.PreferAlignByUnCoreCacheOption: "true",
				},
			}))
		})

		ginkgo.It("should admit container asking odd integer amount of cpus", func(ctx context.Context) {
			// assume uncore caches's worth of cpus will always be an even integer value
			// smallest odd integer cpu request can be 1 cpu
			// for meaningful test, minimum allocatable cpu requirement should be:
			// minCPUCapacity + reservedCPUs.Size() + 1 CPU allocated
			cpuCount := minCPUCapacity + reservedCPUs.Size() + 1
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			// check if the node processor architecture has split or monolithic uncore cache.
			// prefer-align-cpus-by-uncore-cache can be enabled on non-split uncore cache processors
			// with no change to default static behavior
			allocatableCPUs := cpuDetailsFromNode(getLocalNode(ctx, f)).Allocatable
			hasSplitUncore := (allocatableCPUs > int64(uncoreGroupSize))

			if hasSplitUncore {
				// create a container that requires one less cpu than a full uncore cache's worth of cpus
				// assume total shared CPUs of a single uncore cache will always be an even integer
				cpuRequest := fmt.Sprintf("%d000m", (uncoreGroupSize - 1))
				ginkgo.By(fmt.Sprintf("creating the testing pod cpuRequest=%v", cpuRequest))
				pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
					{
						ctnName:    "gu-container-pos",
						cpuRequest: cpuRequest,
						cpuLimit:   cpuRequest,
					},
				})
				ginkgo.By("creating the test pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				podMap[string(pod.UID)] = pod

				ginkgo.By("validating each container in the testing pod")
				for _, cnt := range pod.Spec.Containers {
					ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

					gomega.Expect(pod).To(HaveContainerCPUsWithSameUncoreCacheID(cnt.Name))
				}
			} else {
				// for node with monolithic uncore cache processor
				// uncoreGroupSize will be socket's worth of CPUs
				// subtract (minCPUCapacity + 1) CPU resource constraint
				cpuRequest := fmt.Sprintf("%d000m", (uncoreGroupSize - (minCPUCapacity + 1)))
				ginkgo.By(fmt.Sprintf("creating the testing pod cpuRequest=%v", cpuRequest))
				pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
					{
						ctnName:    "gu-container-pos",
						cpuRequest: cpuRequest,
						cpuLimit:   cpuRequest,
					},
				})
				ginkgo.By("creating the test pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				podMap[string(pod.UID)] = pod

				ginkgo.By("validating each container in the testing pod")
				for _, cnt := range pod.Spec.Containers {
					ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

					gomega.Expect(pod).To(HaveContainerCPUsWithSameUncoreCacheID(cnt.Name))
				}
			}
		})
	})

	ginkgo.When("running with Uncore Cache Alignment disabled", ginkgo.Label("prefer-align-cpus-by-uncore-cache"), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {

			reservedCPUs := cpuset.New(0)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs,
				enableCPUManagerOptions: true,
				options: map[string]string{
					cpumanager.PreferAlignByUnCoreCacheOption: "false",
				},
			}))
		})

		ginkgo.It("should allocate exclusively CPUs to a multi-container pod (1+2)", func(ctx context.Context) {
			cpuCount := 3 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
				{
					ctnName:    "gu-container-1",
					cpuRequest: "1000m",
					cpuLimit:   "1000m",
				},
				{
					ctnName:    "gu-container-2",
					cpuRequest: "2000m",
					cpuLimit:   "2000m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")

			// we cannot nor we should predict which CPUs the container gets
			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 1))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs))

			gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 2))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs))
			// TODO: this is probably too strict but it is the closest of the old test did
			gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container-2"))
		})
	})

	ginkgo.When("checking the compatibility between options", func() {
		// please avoid nesting `BeforeEach` as much as possible. Ideally avoid completely.
		ginkgo.Context("SMT Alignment and strict CPU reservation", ginkgo.Label("smt-alignment", "strict-cpu-reservation"), func() {
			ginkgo.BeforeEach(func(ctx context.Context) {
				// strict SMT alignment is trivially verified and granted on non-SMT systems
				if smtLevel < minSMTLevel {
					e2eskipper.Skipf("Skipping CPU Manager %q tests since SMT disabled", cpumanager.FullPCPUsOnlyOption)
				}
				reservedCPUs = cpuset.New(0)
			})

			ginkgo.It("should reject workload asking non-SMT-multiple of cpus", func(ctx context.Context) {
				cpuCount := 1

				skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount+1) // note the extra for the non-gu pod)

				updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
					policyName:              string(cpumanager.PolicyStatic),
					reservedSystemCPUs:      reservedCPUs,
					enableCPUManagerOptions: true,
					options: map[string]string{
						cpumanager.FullPCPUsOnlyOption:        "true",
						cpumanager.StrictCPUReservationOption: "true",
					},
				}))

				// negative test: try to run a container whose requests aren't a multiple of SMT level, expect a rejection
				pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
					{
						ctnName:    "gu-container-neg",
						cpuRequest: "1000m",
						cpuLimit:   "1000m",
					},
				})
				ginkgo.By("creating the testing pod")
				// CreateSync would wait for pod to become Ready - which will never happen if production code works as intended!
				pod = e2epod.NewPodClient(f).Create(ctx, pod)
				podMap[string(pod.UID)] = pod

				ginkgo.By("ensuring the testing pod is in failed state")
				err := e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "Failed", 30*time.Second, func(pod *v1.Pod) (bool, error) {
					if pod.Status.Phase != v1.PodPending {
						return true, nil
					}
					return false, nil
				})
				framework.ExpectNoError(err)

				ginkgo.By("ensuring the testing pod is failed for the expected reason")
				pod, err = e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
				framework.ExpectNoError(err)
				gomega.Expect(pod).To(BeAPodInPhase(v1.PodFailed))
				gomega.Expect(pod).To(HaveStatusReasonMatchingRegex(`SMT.*Alignment.*Error`))
			})

			ginkgo.It("should admit workload asking SMT-multiple of cpus", func(ctx context.Context) {
				// positive test: try to run a container whose requests are a multiple of SMT level, check allocated cores
				// 1. are core siblings
				// 2. take a full core
				// WARNING: this assumes 2-way SMT systems - we don't know how to access other SMT levels.
				//          this means on more-than-2-way SMT systems this test will prove nothing
				skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), smtLevel)

				updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
					policyName:              string(cpumanager.PolicyStatic),
					reservedSystemCPUs:      reservedCPUs,
					enableCPUManagerOptions: true,
					options: map[string]string{
						cpumanager.FullPCPUsOnlyOption:        "true",
						cpumanager.StrictCPUReservationOption: "true",
					},
				}))

				cpuCount := smtLevel
				cpuRequest := fmt.Sprintf("%d000m", smtLevel)
				ginkgo.By(fmt.Sprintf("creating the testing pod cpuRequest=%v", cpuRequest))
				pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
					{
						ctnName:    "gu-container-x",
						cpuRequest: cpuRequest,
						cpuLimit:   cpuRequest,
					},
				})
				ginkgo.By("creating the testing pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				podMap[string(pod.UID)] = pod

				usableCPUs := onlineCPUs.Difference(reservedCPUs)

				gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-x", cpuCount))
				gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-x", usableCPUs))
				gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-x", reservedCPUs))

				ginkgo.By("validating each container in the testing pod")
				for _, cnt := range pod.Spec.Containers {
					ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

					gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
					gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
				}
			})
		})

		// please avoid nesting `BeforeEach` as much as possible. Ideally avoid completely.
		ginkgo.Context("SMT Alignment and Uncore Cache Alignment", ginkgo.Label("smt-alignment", "prefer-align-cpus-by-uncore-cache"), func() {
			ginkgo.BeforeEach(func(ctx context.Context) {
				// strict SMT alignment is trivially verified and granted on non-SMT systems
				if smtLevel < minSMTLevel {
					e2eskipper.Skipf("Skipping CPU Manager %q tests since SMT disabled", cpumanager.FullPCPUsOnlyOption)
				}
				reservedCPUs = cpuset.New(0)
			})

			ginkgo.It("should assign packed CPUs with prefer-align-cpus-by-uncore-cache disabled and pcpu-only policy options enabled", func(ctx context.Context) {
				skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), smtLevel)

				updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
					policyName:              string(cpumanager.PolicyStatic),
					reservedSystemCPUs:      reservedCPUs,
					enableCPUManagerOptions: true,
					options: map[string]string{
						cpumanager.FullPCPUsOnlyOption:            "true",
						cpumanager.PreferAlignByUnCoreCacheOption: "false",
					},
				}))

				ctnAttrs := []ctnAttribute{
					{
						ctnName:    "test-gu-container-uncore-cache-alignment-disabled",
						cpuRequest: "2000m",
						cpuLimit:   "2000m",
					},
				}
				pod := makeCPUManagerPod("test-pod-uncore-cache-alignment-disabled", ctnAttrs)
				ginkgo.By("creating the test pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				podMap[string(pod.UID)] = pod

				ginkgo.By("validating each container in the testing pod")
				for _, cnt := range pod.Spec.Containers {
					ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

					gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
					gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
				}
			})

			ginkgo.It("should assign CPUs aligned to uncore caches with prefer-align-cpus-by-uncore-cache and pcpu-only policy options enabled", func(ctx context.Context) {

				skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), smtLevel)

				updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
					policyName:              string(cpumanager.PolicyStatic),
					reservedSystemCPUs:      reservedCPUs,
					enableCPUManagerOptions: true,
					options: map[string]string{
						cpumanager.FullPCPUsOnlyOption:            "true",
						cpumanager.PreferAlignByUnCoreCacheOption: "true",
					},
				}))

				// check if the node processor architecture has split or monolithic uncore cache.
				// prefer-align-cpus-by-uncore-cache can be enabled on non-split uncore cache processors
				// with no change to default static behavior
				allocatableCPUs := cpuDetailsFromNode(getLocalNode(ctx, f)).Allocatable
				hasSplitUncore := (allocatableCPUs > int64(uncoreGroupSize))

				if hasSplitUncore {
					// for node with split uncore cache processor
					// create a pod that requires a full uncore cache worth of CPUs
					ctnAttrs := []ctnAttribute{
						{
							ctnName:    "test-gu-container-align-cpus-by-uncore-cache-on-split-uncore",
							cpuRequest: fmt.Sprintf("%d", uncoreGroupSize),
							cpuLimit:   fmt.Sprintf("%d", uncoreGroupSize),
						},
					}
					pod := makeCPUManagerPod("test-pod-align-cpus-by-uncore-cache", ctnAttrs)
					ginkgo.By("creating the test pod")
					pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
					podMap[string(pod.UID)] = pod

					// 'prefer-align-cpus-by-uncore-cache' policy options will attempt at best-effort to allocate cpus
					// so that distribution across uncore caches is minimized. Since the test container is requesting a full
					// uncore cache worth of cpus and CPU0 is part of the reserved CPUset and not allocatable, the policy will attempt
					// to allocate cpus from the next available uncore cache

					for _, cnt := range pod.Spec.Containers {
						ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

						gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
						cpus, err := getContainerAllowedCPUs(pod, cnt.Name, false)
						framework.ExpectNoError(err, "cannot get cpus allocated to pod %s/%s cnt %s", pod.Namespace, pod.Name, cnt.Name)

						siblingsCPUs := makeThreadSiblingCPUSet(cpus)
						gomega.Expect(pod).To(HaveContainerCPUsEqualTo(cnt.Name, siblingsCPUs))

						gomega.Expect(pod).To(HaveContainerCPUsWithSameUncoreCacheID(cnt.Name))
						gomega.Expect(pod).ToNot(HaveContainerCPUsShareUncoreCacheWith(cnt.Name, reservedCPUs))
					}
				} else {
					// for node with monolithic uncore cache processor
					// expect default static behavior with pcpu-only policy enabled
					// and prefer-align-cpus-by-uncore-cache enabled
					ctnAttrs := []ctnAttribute{
						{
							ctnName:    "test-gu-container-align-cpus-by-uncore-cache-on-mono-uncore",
							cpuRequest: "2000m",
							cpuLimit:   "2000m",
						},
					}
					pod := makeCPUManagerPod("test-pod-align-cpus-by-uncore-cache", ctnAttrs)
					ginkgo.By("creating the test pod")
					pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
					podMap[string(pod.UID)] = pod

					ginkgo.By("validating each container in the testing pod")
					for _, cnt := range pod.Spec.Containers {
						ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

						gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
						gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
					}
				}
			})
		})

		// please avoid nesting `BeforeEach` as much as possible. Ideally avoid completely.
		ginkgo.Context("Strict CPU Reservation and Uncore Cache Alignment", ginkgo.Label("strict-cpu-reservation", "prefer-align-cpus-by-uncore-cache"), func() {
			ginkgo.BeforeEach(func(ctx context.Context) {
				reservedCPUs = cpuset.New(0)
			})

			ginkgo.It("should assign CPUs aligned to uncore caches with prefer-align-cpus-by-uncore-cache and avoid reserved cpus", func(ctx context.Context) {
				// assume uncore caches's worth of cpus will always be an even integer value
				// smallest integer cpu request can be 1 cpu
				// for meaningful test, minimum allocatable cpu requirement should be:
				// minCPUCapacity + reservedCPUs.Size() + 1 CPU allocated
				cpuCount := minCPUCapacity + reservedCPUs.Size() + 1
				skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

				updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
					policyName:              string(cpumanager.PolicyStatic),
					reservedSystemCPUs:      reservedCPUs,
					enableCPUManagerOptions: true,
					options: map[string]string{
						cpumanager.StrictCPUReservationOption:     "true",
						cpumanager.PreferAlignByUnCoreCacheOption: "true",
					},
				}))

				// check if the node processor architecture has split or monolithic uncore cache.
				// prefer-align-cpus-by-uncore-cache can be enabled on non-split uncore cache processors
				// with no change to default static behavior
				allocatableCPUs := cpuDetailsFromNode(getLocalNode(ctx, f)).Allocatable
				hasSplitUncore := (allocatableCPUs > int64(uncoreGroupSize))

				if hasSplitUncore {
					// for node with split uncore cache processor
					// create a pod that requires a full uncore cache worth of CPUs
					ctnAttrs := []ctnAttribute{
						{
							ctnName:    "test-gu-container-align-cpus-by-uncore-cache-on-split-uncore",
							cpuRequest: fmt.Sprintf("%d", uncoreGroupSize),
							cpuLimit:   fmt.Sprintf("%d", uncoreGroupSize),
						},
					}
					pod := makeCPUManagerPod("test-pod-align-cpus-by-uncore-cache", ctnAttrs)
					ginkgo.By("creating the test pod")
					pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
					podMap[string(pod.UID)] = pod

					// 'prefer-align-cpus-by-uncore-cache' policy options will attempt at best-effort to allocate cpus
					// so that distribution across uncore caches is minimized. Since the test container is requesting a full
					// uncore cache worth of cpus and CPU0 is part of the reserved CPUset and not allocatable, the policy will attempt
					// to allocate cpus from the next available uncore cache

					for _, cnt := range pod.Spec.Containers {
						ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

						gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
						cpus, err := getContainerAllowedCPUs(pod, cnt.Name, false)
						framework.ExpectNoError(err, "cannot get cpus allocated to pod %s/%s cnt %s", pod.Namespace, pod.Name, cnt.Name)

						siblingsCPUs := makeThreadSiblingCPUSet(cpus)
						gomega.Expect(pod).To(HaveContainerCPUsEqualTo(cnt.Name, siblingsCPUs))

						gomega.Expect(pod).To(HaveContainerCPUsWithSameUncoreCacheID(cnt.Name))
						gomega.Expect(pod).ToNot(HaveContainerCPUsShareUncoreCacheWith(cnt.Name, reservedCPUs))
					}
				} else {
					// for node with monolithic uncore cache processor
					// expect default static packed behavior
					// when prefer-align-cpus-by-uncore-cache enabled
					ctnAttrs := []ctnAttribute{
						{
							ctnName:    "test-gu-container-align-cpus-by-uncore-cache-on-mono-uncore",
							cpuRequest: "1000m",
							cpuLimit:   "1000m",
						},
					}
					pod := makeCPUManagerPod("test-pod-align-cpus-by-uncore-cache", ctnAttrs)
					ginkgo.By("creating the test pod")
					pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
					podMap[string(pod.UID)] = pod

					ginkgo.By("validating each container in the testing pod")
					for _, cnt := range pod.Spec.Containers {
						ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

						// expect allocated CPUs to be on the same uncore cache
						gomega.Expect(pod).To(HaveContainerCPUsWithSameUncoreCacheID(cnt.Name))
						gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("test-gu-container-align-cpus-by-uncore-cache-on-mono-uncore", reservedCPUs))
					}
				}
			})
		})

		// please avoid nesting `BeforeEach` as much as possible. Ideally avoid completely.
		ginkgo.Context("SMT Alignment and distribution across NUMA", ginkgo.Label("smt-alignment", "distribute-cpus-across-numa"), func() {
			ginkgo.BeforeEach(func(ctx context.Context) {
				// strict SMT alignment is trivially verified and granted on non-SMT systems
				if smtLevel < minSMTLevel {
					e2eskipper.Skipf("Skipping CPU Manager %q tests since SMT disabled", cpumanager.FullPCPUsOnlyOption)
				}
				reservedCPUs = cpuset.New(0)
			})

			ginkgo.It("should assign packed CPUs with distribute-cpus-across-numa disabled and pcpu-only policy options enabled", func(ctx context.Context) {
				skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), smtLevel)

				updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
					policyName:              string(cpumanager.PolicyStatic),
					reservedSystemCPUs:      reservedCPUs,
					enableCPUManagerOptions: true,
					options: map[string]string{
						cpumanager.FullPCPUsOnlyOption:            "true",
						cpumanager.DistributeCPUsAcrossNUMAOption: "false",
					},
				}))

				ctnAttrs := []ctnAttribute{
					{
						ctnName:    "test-gu-container-distribute-cpus-across-numa-disabled",
						cpuRequest: "2000m",
						cpuLimit:   "2000m",
					},
				}
				pod := makeCPUManagerPod("test-pod-distribute-cpus-across-numa-disabled", ctnAttrs)
				ginkgo.By("creating the test pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				podMap[string(pod.UID)] = pod

				ginkgo.By("validating each container in the testing pod")
				for _, cnt := range pod.Spec.Containers {
					ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

					gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
					gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
				}
			})

			ginkgo.It("should assign CPUs distributed across NUMA with distribute-cpus-across-numa and pcpu-only policy options enabled", func(ctx context.Context) {
				reservedCPUs = cpuset.New(0)

				// this test is intended to be run on a multi-node NUMA system and
				// a system with at least 4 cores per socket, hostcheck skips test
				// if above requirements are not satisfied
				numaNodeNum, _, _, cpusNumPerNUMA := hostCheck()
				cpuReq := (cpusNumPerNUMA - smtLevel) * numaNodeNum

				skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuReq)

				updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
					policyName:              string(cpumanager.PolicyStatic),
					reservedSystemCPUs:      reservedCPUs,
					enableCPUManagerOptions: true,
					options: map[string]string{
						cpumanager.FullPCPUsOnlyOption:            "true",
						cpumanager.DistributeCPUsAcrossNUMAOption: "true",
					},
				}))

				// 'distribute-cpus-across-numa' policy option ensures that CPU allocations are evenly distributed
				//  across NUMA nodes in cases where more than one NUMA node is required to satisfy the allocation.
				// So, we want to ensure that the CPU Request exceeds the number of CPUs that can fit within a single
				// NUMA node. We have to pick cpuRequest such that:
				// 1. CPURequest > cpusNumPerNUMA
				// 2. Not occupy all the CPUs on the node ande leave room for reserved CPU
				// 3. CPURequest is a multiple if number of NUMA nodes to allow equal CPU distribution across NUMA nodes
				//
				// In summary: cpusNumPerNUMA < CPURequest < ((cpusNumPerNuma * numaNodeNum) - reservedCPUscount)
				// Considering all these constraints we select: CPURequest= (cpusNumPerNUMA-smtLevel)*numaNodeNum

				ctnAttrs := []ctnAttribute{
					{
						ctnName:    "test-gu-container-distribute-cpus-across-numa",
						cpuRequest: fmt.Sprintf("%d", cpuReq),
						cpuLimit:   fmt.Sprintf("%d", cpuReq),
					},
				}
				pod := makeCPUManagerPod("test-pod-distribute-cpus-across-numa", ctnAttrs)
				ginkgo.By("creating the test pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				podMap[string(pod.UID)] = pod

				for _, cnt := range pod.Spec.Containers {
					ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))

					gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
					cpus, err := getContainerAllowedCPUs(pod, cnt.Name, false)
					framework.ExpectNoError(err, "cannot get cpus allocated to pod %s/%s cnt %s", pod.Namespace, pod.Name, cnt.Name)

					siblingsCPUs := makeThreadSiblingCPUSet(cpus)
					gomega.Expect(pod).To(HaveContainerCPUsEqualTo(cnt.Name, siblingsCPUs))

					// We expect a perfectly even spilit i.e. equal distribution across NUMA Node as the CPU Request is 4*smtLevel*numaNodeNum.
					expectedSpread := cpus.Size() / numaNodeNum
					gomega.Expect(cpus).To(BeDistributedCPUs(expectedSpread))
				}
			})
		})
	})

	ginkgo.When("checking the CFS quota management", ginkgo.Label("cfs-quota"), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {
			requireCGroupV2()
			// WARNING: this assumes 2-way SMT systems - we don't know how to access other SMT levels.
			//          this means on more-than-2-way SMT systems this test will prove nothing
			reservedCPUs = cpuset.New(0)
			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:                       string(cpumanager.PolicyStatic),
				reservedSystemCPUs:               reservedCPUs,
				disableCPUQuotaWithExclusiveCPUs: true,
			}))
		})

		ginkgo.It("should enforce for best-effort pod", func(ctx context.Context) {
			ctnName := "be-container"
			pod := makeCPUManagerBEPod("be-pod", []ctnAttribute{
				{
					ctnName:    ctnName,
					ctnCommand: "sleep 1d",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("max"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "max"))
		})

		ginkgo.It("should disable for guaranteed pod with exclusive single CPU assigned", func(ctx context.Context) {
			cpuCount := 1
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			ctnName := "gu-container-cfsquota-disabled"
			pod := makeCPUManagerPod("gu-pod-cfsquota-off", []ctnAttribute{
				{
					ctnName:    ctnName,
					cpuRequest: "1",
					cpuLimit:   "1",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("max"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "max"))
		})

		ginkgo.It("should disable for guaranteed pod with exclusive four CPUs assigned", func(ctx context.Context) {
			cpuCount := 4
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			ctnName := "gu-container-cfsquota-disabled"
			pod := makeCPUManagerPod("gu-pod-cfsquota-off", []ctnAttribute{
				{
					ctnName:    ctnName,
					cpuRequest: "3",
					cpuLimit:   "3",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("max"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "max"))

			gomega.Expect(pod).To(HaveContainerCPUsCount(ctnName, 3))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf(ctnName, onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith(ctnName, reservedCPUs))
		})

		ginkgo.It("should enforce for guaranteed pod", func(ctx context.Context) {
			cpuCount := 1 // overshoot, minimum request is 1
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			ctnName := "gu-container-cfsquota-enabled"
			pod := makeCPUManagerPod("gu-pod-cfs-quota-on", []ctnAttribute{
				{
					ctnName:    ctnName,
					cpuRequest: "500m",
					cpuLimit:   "500m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("50000"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "50000"))
		})

		ginkgo.It("should enforce for burstable pod", func(ctx context.Context) {
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 0)

			ctnName := "bu-container-cfsquota-enabled"
			pod := makeCPUManagerPod("bu-pod-cfs-quota-on", []ctnAttribute{
				{
					ctnName:    ctnName,
					cpuRequest: "100m",
					cpuLimit:   "500m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("50000"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "50000"))
		})

		ginkgo.It("should not enforce with multiple containers without exclusive CPUs", func(ctx context.Context) {
			cpuCount := 2
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			pod := makeCPUManagerPod("gu-pod-multicontainer", []ctnAttribute{
				{
					ctnName:    "gu-container-non-int-values-1",
					cpuRequest: "100m",
					cpuLimit:   "500m",
				},
				{
					ctnName:    "gu-container-non-int-values-2",
					cpuRequest: "300m",
					cpuLimit:   "1200m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("170000"))
			gomega.Expect(pod).To(HaveContainerQuota("gu-container-non-int-values-1", "50000"))
			gomega.Expect(pod).To(HaveContainerQuota("gu-container-non-int-values-2", "120000"))
		})

		ginkgo.It("should not enforce with multiple containers only in the container with exclusive CPUs", func(ctx context.Context) {
			cpuCount := 2
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			pod := makeCPUManagerPod("gu-pod-multicontainer-mixed", []ctnAttribute{
				{
					ctnName:    "gu-container-non-int-values",
					cpuRequest: "500m",
					cpuLimit:   "500m",
				},
				{
					ctnName:    "gu-container-int-values",
					cpuRequest: "1",
					cpuLimit:   "1",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("max"))
			gomega.Expect(pod).To(HaveContainerQuota("gu-container-non-int-values", "50000"))
			gomega.Expect(pod).To(HaveContainerQuota("gu-container-int-values", "max"))
		})
	})

	ginkgo.When("checking the CFS quota management can be disabled", ginkgo.Label("cfs-quota"), func() {
		// NOTE: these tests check only cases on which the quota is set to "max", so we intentionally
		// don't duplicate the all the tests

		ginkgo.BeforeEach(func(ctx context.Context) {
			requireCGroupV2()
			// WARNING: this assumes 2-way SMT systems - we don't know how to access other SMT levels.
			//          this means on more-than-2-way SMT systems this test will prove nothing
			reservedCPUs = cpuset.New(0)
			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:                       string(cpumanager.PolicyStatic),
				reservedSystemCPUs:               reservedCPUs,
				disableCPUQuotaWithExclusiveCPUs: false,
			}))
		})

		ginkgo.It("should enforce for a guaranteed pod with exclusive CPUs assigned", func(ctx context.Context) {
			cpuCount := 1
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			ctnName := "gu-container-cfsquota-disabled"
			pod := makeCPUManagerPod("gu-pod-cfsquota-off", []ctnAttribute{
				{
					ctnName:    ctnName,
					cpuRequest: "1",
					cpuLimit:   "1",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("100000"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "100000"))
		})

		ginkgo.It("should enforce for a guaranteed pod with multiple exclusive CPUs assigned", func(ctx context.Context) {
			cpuCount := 4
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			ctnName := "gu-container-cfsquota-disabled"
			pod := makeCPUManagerPod("gu-pod-cfsquota-off", []ctnAttribute{
				{
					ctnName:    ctnName,
					cpuRequest: "3",
					cpuLimit:   "3",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("300000"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "300000"))

			gomega.Expect(pod).To(HaveContainerCPUsCount(ctnName, 3))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf(ctnName, onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith(ctnName, reservedCPUs))
		})

		ginkgo.It("should enforce for guaranteed pod not requiring exclusive CPUs", func(ctx context.Context) {
			cpuCount := 1 // overshoot, minimum request is 1
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			ctnName := "gu-container-cfsquota-enabled"
			pod := makeCPUManagerPod("gu-pod-cfs-quota-on", []ctnAttribute{
				{
					ctnName:    ctnName,
					cpuRequest: "500m",
					cpuLimit:   "500m",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("50000"))
			gomega.Expect(pod).To(HaveContainerQuota(ctnName, "50000"))
		})

		ginkgo.It("should enforce with multiple containers regardless if they require exclusive CPUs or not", func(ctx context.Context) {
			cpuCount := 2
			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			pod := makeCPUManagerPod("gu-pod-multicontainer-mixed", []ctnAttribute{
				{
					ctnName:    "gu-container-non-int-values",
					cpuRequest: "500m",
					cpuLimit:   "500m",
				},
				{
					ctnName:    "gu-container-int-values",
					cpuRequest: "1",
					cpuLimit:   "1",
				},
			})
			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			gomega.Expect(pod).To(HaveSandboxQuota("150000"))
			gomega.Expect(pod).To(HaveContainerQuota("gu-container-non-int-values", "50000"))
			gomega.Expect(pod).To(HaveContainerQuota("gu-container-int-values", "100000"))
		})
	})

	f.Context("When checking the sidecar containers", framework.WithNodeConformance(), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {
			reservedCPUs = cpuset.New(0)
		})

		ginkgo.It("should reuse init container exclusive CPUs, but not sidecar container exclusive CPUS", func(ctx context.Context) {
			cpuCount := 2 // total

			skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), cpuCount)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:         string(cpumanager.PolicyStatic),
				reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
			}))

			var containerRestartPolicyAlways = v1.ContainerRestartPolicyAlways

			ctrAttrs := []ctnAttribute{
				{
					ctnName:    "init-container1",
					cpuRequest: "1000m",
					cpuLimit:   "1000m",
				},
				{
					ctnName:       "sidecar-container",
					cpuRequest:    "1000m",
					cpuLimit:      "1000m",
					restartPolicy: &containerRestartPolicyAlways,
				},
			}
			pod := makeCPUManagerInitContainersPod("gu-pod", ctrAttrs)
			ginkgo.By("running a Gu pod with a regular init container and a restartable init container")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			// when we get there the real initcontainer terminated, so we can only check its logs
			ginkgo.By("checking if the expected cpuset was assigned")
			logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, pod.Namespace, pod.Name, pod.Spec.InitContainers[0].Name)
			framework.ExpectNoError(err, "expected log not found in init container [%s] of pod [%s]", pod.Spec.InitContainers[0].Name, pod.Name)

			reusableCPUs := getContainerAllowedCPUsFromLogs(pod.Name, pod.Spec.InitContainers[0].Name, logs)
			gomega.Expect(reusableCPUs.Size()).To(gomega.Equal(1), "expected cpu set size == 1, got %q", reusableCPUs.String())

			nonReusableCPUs, err := getContainerAllowedCPUs(pod, pod.Spec.InitContainers[1].Name, true)
			framework.ExpectNoError(err, "cannot get exclusive CPUs for pod %s/%s", pod.Namespace, pod.Name)

			gomega.Expect(nonReusableCPUs.Size()).To(gomega.Equal(1), "expected cpu set size == 1, got %q", nonReusableCPUs.String())
			gomega.Expect(reusableCPUs.Equals(nonReusableCPUs)).To(gomega.BeTrueBecause("expected reusable cpuset [%s] to be equal to non-reusable cpuset [%s]", reusableCPUs.String(), nonReusableCPUs.String()))

			appContainerName := pod.Spec.Containers[0].Name
			gomega.Expect(pod).To(HaveContainerCPUsCount(appContainerName, 1))
			gomega.Expect(pod).To(HaveContainerCPUsASubsetOf(appContainerName, onlineCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith(appContainerName, reservedCPUs))
			gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith(appContainerName, nonReusableCPUs))
		})
	})
})

var _ = SIGDescribe("CPU Manager Incompatibility Pod Level Resources", ginkgo.Ordered, ginkgo.ContinueOnFailure, framework.WithSerial(), feature.CPUManager, feature.PodLevelResources, framework.WithFeatureGate(features.PodLevelResources), func() {
	f := framework.NewDefaultFramework("cpu-manager-incompatibility-pod-level-resources-test")
	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged

	// original kubeletconfig before the context start, to be restored
	var oldCfg *kubeletconfig.KubeletConfiguration
	var reservedCPUs cpuset.CPUSet
	var onlineCPUs cpuset.CPUSet
	var smtLevel int
	var uncoreGroupSize int
	// tracks all the pods created by a It() block. Best would be a namespace per It block
	var podMap map[string]*v1.Pod

	ginkgo.BeforeAll(func(ctx context.Context) {
		var err error
		oldCfg, err = getCurrentKubeletConfig(ctx)
		framework.ExpectNoError(err)

		onlineCPUs, err = getOnlineCPUs() // this should not change at all, at least during this suite lifetime
		framework.ExpectNoError(err)
		framework.Logf("Online CPUs: %s", onlineCPUs)

		smtLevel = smtLevelFromSysFS() // this should not change at all, at least during this suite lifetime
		framework.Logf("SMT level: %d", smtLevel)

		uncoreGroupSize = getUncoreCPUGroupSize()
		framework.Logf("Uncore Group Size: %d", uncoreGroupSize)

		e2enodeCgroupV2Enabled = IsCgroup2UnifiedMode()
		framework.Logf("cgroup V2 enabled: %v", e2enodeCgroupV2Enabled)

		e2enodeCgroupDriver = oldCfg.CgroupDriver
		framework.Logf("cgroup driver: %s", e2enodeCgroupDriver)

		runtime, _, err := getCRIClient()
		framework.ExpectNoError(err, "Failed to get CRI client")

		version, err := runtime.Version(context.Background(), "")
		framework.ExpectNoError(err, "Failed to get runtime version")

		e2enodeRuntimeName = version.GetRuntimeName()
		framework.Logf("runtime: %s", e2enodeRuntimeName)
	})

	ginkgo.AfterAll(func(ctx context.Context) {
		updateKubeletConfig(ctx, f, oldCfg, true)
	})

	ginkgo.BeforeEach(func(ctx context.Context) {
		// note intentionally NOT set reservedCPUs -  this must be initialized on a test-by-test basis
		podMap = make(map[string]*v1.Pod)
	})

	ginkgo.JustBeforeEach(func(ctx context.Context) {
		if !e2enodeCgroupV2Enabled {
			e2eskipper.Skipf("Skipping since CgroupV2 not used")
		}
	})

	ginkgo.AfterEach(func(ctx context.Context) {
		deletePodsAsync(ctx, f, podMap)
	})

	ginkgo.When("running guaranteed pod level resources tests", ginkgo.Label("guaranteed pod level resources", "reserved-cpus"), func() {
		ginkgo.It("should let the container access all the online CPUs without a reserved CPUs set", func(ctx context.Context) {
			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      cpuset.CPUSet{},
				enablePodLevelResources: true,
			}))

			pod := makeCPUManagerPod("gu-pod-level-resources", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: "1",
					cpuLimit:   "1",
				},
			})
			pod.Spec.Resources = &v1.ResourceRequirements{
				Limits: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("1"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("1"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			}

			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")
			gomega.Expect(pod).To(HaveContainerCPUsEqualTo("gu-container", onlineCPUs))
		})

		ginkgo.It("should let the container access all the online CPUs when using a reserved CPUs set", func(ctx context.Context) {
			reservedCPUs = cpuset.New(0)

			updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
				policyName:              string(cpumanager.PolicyStatic),
				reservedSystemCPUs:      reservedCPUs, // Not really needed for the tests but helps to make a more precise check
				enablePodLevelResources: true,
			}))

			pod := makeCPUManagerPod("gu-pod-level-resources", []ctnAttribute{
				{
					ctnName:    "gu-container",
					cpuRequest: "1",
					cpuLimit:   "1",
				},
			})
			pod.Spec.Resources = &v1.ResourceRequirements{
				Limits: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("1"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("1"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			}

			ginkgo.By("creating the test pod")
			pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
			podMap[string(pod.UID)] = pod

			ginkgo.By("checking if the expected cpuset was assigned")
			gomega.Expect(pod).To(HaveContainerCPUsEqualTo("gu-container", onlineCPUs))
		})
	})
})

// Matching helpers

func HaveStatusReasonMatchingRegex(expr string) types.GomegaMatcher {
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		re, err := regexp.Compile(expr)
		if err != nil {
			return false, err
		}
		return re.MatchString(actual.Status.Reason), nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} reason {{.Actual.Status.Reason}} does not match regexp {{.Data}}", expr)
}

type msgData struct {
	Name             string
	CurrentCPUs      string
	ExpectedCPUs     string
	MismatchedCPUs   string
	UncoreCacheAlign string
	Count            int
	Aligned          int
	CurrentQuota     string
	ExpectedQuota    string
}

func HaveContainerCPUsCount(ctnName string, val int) types.GomegaMatcher {
	md := &msgData{
		Name:  ctnName,
		Count: val,
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		md.CurrentCPUs = cpus.String()
		if err != nil {
			framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
			return false, err
		}
		return cpus.Size() == val, nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not matching expected count <{{.Data.Count}}> for container {{.Data.Name}}", md)
}

func HaveContainerCPUsAlignedTo(ctnName string, val int) types.GomegaMatcher {
	md := &msgData{
		Name:    ctnName,
		Aligned: val,
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		md.CurrentCPUs = cpus.String()
		if err != nil {
			framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
			return false, err
		}
		return cpus.Size()%val == 0, nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not aligned to value <{{.Data.Aligned}}> for container {{.Data.Name}}", md)
}

func HaveContainerCPUsOverlapWith(ctnName string, ref cpuset.CPUSet) types.GomegaMatcher {
	md := &msgData{
		Name:         ctnName,
		ExpectedCPUs: ref.String(),
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		md.CurrentCPUs = cpus.String()
		if err != nil {
			framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
			return false, err
		}
		sharedCPUs := cpus.Intersection(ref)
		return sharedCPUs.Size() > 0, nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> overlapping with expected CPUs <{{.Data.ExpectedCPUs}}> for container {{.Data.Name}}", md)
}

func HaveContainerCPUsASubsetOf(ctnName string, ref cpuset.CPUSet) types.GomegaMatcher {
	md := &msgData{
		Name:         ctnName,
		ExpectedCPUs: ref.String(),
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		md.CurrentCPUs = cpus.String()
		if err != nil {
			framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
			return false, err
		}
		return cpus.IsSubsetOf(ref), nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not a subset of expected CPUs <{{.Data.ExpectedCPUs}}> for container {{.Data.Name}}", md)
}

func HaveContainerCPUsEqualTo(ctnName string, expectedCPUs cpuset.CPUSet) types.GomegaMatcher {
	md := &msgData{
		Name:         ctnName,
		ExpectedCPUs: expectedCPUs.String(),
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		md.CurrentCPUs = cpus.String()
		if err != nil {
			framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
			return false, err
		}
		return cpus.Equals(expectedCPUs), nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not matching the expected value <{{.Data.ExpectedCPUs}}> for container {{.Data.Name}}", md)
}

func HaveSandboxQuota(expectedQuota string) types.GomegaMatcher {
	md := &msgData{
		ExpectedQuota: expectedQuota,
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		md.Name = klog.KObj(actual).String()
		quota, err := getSandboxCFSQuota(actual)
		md.CurrentQuota = quota
		if err != nil {
			framework.Logf("getSandboxCFSQuota() failed: %v", err)
			return false, err
		}
		re, err := regexp.Compile(fmt.Sprintf("^%s %s$", expectedQuota, defaultCFSPeriod))
		if err != nil {
			return false, err
		}
		return re.MatchString(quota), nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has quota <{{.Data.CurrentQuota}}> not matching expected value <{{.Data.ExpectedQuota}}>", md)
}

func HaveContainerQuota(ctnName, expectedQuota string) types.GomegaMatcher {
	md := &msgData{
		Name:          ctnName,
		ExpectedQuota: expectedQuota,
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		quota, err := getContainerCFSQuota(actual, ctnName, false)
		md.CurrentQuota = quota
		if err != nil {
			framework.Logf("getContainerCFSQuota(%s) failed: %v", ctnName, err)
			return false, err
		}
		re, err := regexp.Compile(fmt.Sprintf("^%s %s$", expectedQuota, defaultCFSPeriod))
		if err != nil {
			return false, err
		}
		return re.MatchString(quota), nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has quota <{{.Data.CurrentQuota}}> not matching expected value <{{.Data.ExpectedQuota}}> for container {{.Data.Name}}", md)
}

func HaveContainerCPUsThreadSiblings(ctnName string) types.GomegaMatcher {
	md := &msgData{
		Name: ctnName,
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		md.CurrentCPUs = cpus.String()
		if err != nil {
			framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
			return false, err
		}
		expectedCPUs := makeThreadSiblingCPUSet(cpus)
		md.ExpectedCPUs = expectedCPUs.String()
		return cpus.Equals(expectedCPUs), nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not all thread sibling pairs (would be <{{.Data.ExpectedCPUs}}>) for container {{.Data.Name}}", md)
}

func HaveContainerCPUsQuasiThreadSiblings(ctnName string, toleration int) types.GomegaMatcher {
	md := &msgData{
		Name:  ctnName,
		Count: toleration,
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		md.CurrentCPUs = cpus.String()
		if err != nil {
			framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
			return false, err
		}
		// this is by construction >= cpus (extreme case: cpus is made by all non-thread-siblings)
		expectedCPUs := makeThreadSiblingCPUSet(cpus)
		md.ExpectedCPUs = expectedCPUs.String()
		mismatchedCPUs := expectedCPUs.Difference(cpus)
		md.MismatchedCPUs = mismatchedCPUs.String()
		return mismatchedCPUs.Size() <= toleration, nil
	}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not all thread sibling pairs (would be <{{.Data.ExpectedCPUs}}> mismatched <{{.Data.MismatchedCPUs}}> toleration <{{.Data.Count}}>) for container {{.Data.Name}}", md)
}

func HaveContainerCPUsWithSameUncoreCacheID(ctnName string) types.GomegaMatcher {
	md := &msgData{
		Name: ctnName,
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		cpus, err := getContainerAllowedCPUs(actual, ctnName, false)
		if err != nil {
			return false, fmt.Errorf("getContainerAllowedCPUs(%s) failed: %w", ctnName, err)
		}
		md.CurrentCPUs = cpus.String()

		var commonCacheID *int64

		for _, cpu := range cpus.List() {
			// determine the Uncore Cache ID for each cpu
			uncoreID, err := uncoreCacheIDFromSysFS(cpu)
			if err != nil {
				return false, fmt.Errorf("failed to read cache ID for CPU %d: %w", cpu, err)
			}

			// if this the first CPU we check, set the Uncore Cache ID as the reference
			// for subsequent CPUs, compare the Uncore Cache ID to the reference
			if commonCacheID == nil {
				commonCacheID = &uncoreID
			} else if *commonCacheID != uncoreID {
				md.UncoreCacheAlign = fmt.Sprintf("shared uncoreID mismatch: CPU %d has uncoreID %d, CPUSet has uncoreID %d", cpu, uncoreID, *commonCacheID)
				return false, nil
			}
		}

		// All CPUs matched the same cache ID
		md.UncoreCacheAlign = fmt.Sprintf("all CPUs share cache ID %d", *commonCacheID)
		return true, nil
	}).WithTemplate(
		"Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} container {{.Data.Name}} has CPUSet <{{.Data.CurrentCPUs}}> where not all CPUs share the same uncore cache ID: {{.Data.UncoreCacheAlign}}",
		md,
	)
}

func HaveContainerCPUsShareUncoreCacheWith(ctnName string, ref cpuset.CPUSet) types.GomegaMatcher {
	md := &msgData{
		Name:         ctnName,
		ExpectedCPUs: ref.String(),
	}
	return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
		containerCPUs, err := getContainerAllowedCPUs(actual, ctnName, false)
		if err != nil {
			return false, fmt.Errorf("getContainerAllowedCPUs(%s) failed: %w", ctnName, err)
		}
		md.CurrentCPUs = containerCPUs.String()

		// Build set of uncore cache IDs from the reference cpuset
		refUncoreIDs := sets.New[int64]()
		for _, cpu := range ref.UnsortedList() {
			uncoreID, err := uncoreCacheIDFromSysFS(cpu)
			if err != nil {
				return false, fmt.Errorf("failed to read uncore cache ID for reference CPU %d: %w", cpu, err)
			}
			refUncoreIDs.Insert(uncoreID)
		}

		// Check if any container CPUs share an uncore ID with the reference set
		for _, cpu := range containerCPUs.UnsortedList() {
			uncoreID, err := uncoreCacheIDFromSysFS(cpu)
			if err != nil {
				return false, fmt.Errorf("failed to read uncore cache ID for container CPU %d: %w", cpu, err)
			}
			if refUncoreIDs.Has(uncoreID) {
				md.UncoreCacheAlign = fmt.Sprintf("%d", uncoreID)
				return true, nil
			}
		}

		return false, nil
	}).WithTemplate(
		"Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} container {{.Data.Name}} has CPUSet <{{.Data.CurrentCPUs}}> sharing uncoreCache ID <{{.Data.UncoreCacheAlign}}> with reference CPUSet <{{.Data.ExpectedCPUs}}>",
		md,
	)
}

// Custom matcher for checking packed CPUs.
func BePackedCPUs() types.GomegaMatcher {
	return gcustom.MakeMatcher(func(allocatedCPUs cpuset.CPUSet) (bool, error) {
		distribution := computeNUMADistribution(allocatedCPUs)
		for _, count := range distribution {
			// This assumption holds true if there are enough CPUs on a single NUMA node.
			// We are intentionally limiting the CPU request to 2 to minimize the number
			// of CPUs required to fulfill this case and therefore maximize the chances
			// of correctly validating this case.
			if count == allocatedCPUs.Size() {
				return true, nil
			}
		}
		return false, nil
	}).WithMessage("expected CPUs to be packed")
}

// Custom matcher for checking distributed CPUs.
func BeDistributedCPUs(expectedSpread int) types.GomegaMatcher {
	return gcustom.MakeMatcher(func(allocatedCPUs cpuset.CPUSet) (bool, error) {
		distribution := computeNUMADistribution(allocatedCPUs)
		for _, count := range distribution {
			if count != expectedSpread {
				return false, nil
			}
		}
		return true, nil
	}).WithTemplate("expected CPUs to be evenly distributed across NUMA nodes\nExpected: {{.Data}}\nGot:\n{{.FormattedActual}}\nDistribution: {{.Data}}\n").WithTemplateData(expectedSpread)
}

// Other helpers

func getContainerAllowedCPUsFromLogs(podName, cntName, logs string) cpuset.CPUSet {
	framework.Logf("got pod logs: <%v>", logs)
	cpus, err := cpuset.Parse(strings.TrimSpace(logs))
	framework.ExpectNoError(err, "parsing cpuset from logs for [%s] of pod [%s]", cntName, podName)
	return cpus
}

// computeNUMADistribution calculates CPU distribution per NUMA node.
func computeNUMADistribution(allocatedCPUs cpuset.CPUSet) map[int]int {
	numaCPUs, err := getNumaNodeCPUs()
	framework.ExpectNoError(err, "Error retrieving NUMA nodes")
	framework.Logf("NUMA Node CPUs allocation: %v", numaCPUs)

	distribution := make(map[int]int)
	for node, cpus := range numaCPUs {
		distribution[node] = cpus.Intersection(allocatedCPUs).Size()
	}

	framework.Logf("allocated CPUs %s distribution: %v", allocatedCPUs.String(), distribution)
	return distribution
}

func getContainerAllowedCPUs(pod *v1.Pod, ctnName string, isInit bool) (cpuset.CPUSet, error) {
	cgPath, err := makeCgroupPathForContainer(pod, ctnName, isInit, e2enodeCgroupV2Enabled)
	if err != nil {
		return cpuset.CPUSet{}, err
	}
	cgPath = filepath.Join(cgPath, cpusetFileNameFromVersion(e2enodeCgroupV2Enabled))
	framework.Logf("pod %s/%s cnt %s qos=%s path %q", pod.Namespace, pod.Name, ctnName, pod.Status.QOSClass, cgPath)
	data, err := os.ReadFile(cgPath)
	if err != nil {
		return cpuset.CPUSet{}, err
	}
	cpus := strings.TrimSpace(string(data))
	framework.Logf("pod %s/%s cnt %s cpuset %q", pod.Namespace, pod.Name, ctnName, cpus)
	return cpuset.Parse(cpus)
}

func getSandboxCFSQuota(pod *v1.Pod) (string, error) {
	if !e2enodeCgroupV2Enabled {
		return "", fmt.Errorf("only Cgroup V2 is supported")
	}
	cgPath := filepath.Join(makeCgroupPathForPod(pod, true), "cpu.max")
	data, err := os.ReadFile(cgPath)
	if err != nil {
		return "", err
	}
	quota := strings.TrimSpace(string(data))
	framework.Logf("pod %s/%s qos=%s path %q quota %q", pod.Namespace, pod.Name, pod.Status.QOSClass, cgPath, quota)
	return quota, nil
}

func getContainerCFSQuota(pod *v1.Pod, ctnName string, isInit bool) (string, error) {
	if !e2enodeCgroupV2Enabled {
		return "", fmt.Errorf("only Cgroup V2 is supported")
	}
	cgPath, err := makeCgroupPathForContainer(pod, ctnName, isInit, true)
	if err != nil {
		return "", err
	}
	data, err := os.ReadFile(filepath.Join(cgPath, "cpu.max"))
	if err != nil {
		return "", err
	}
	quota := strings.TrimSpace(string(data))
	framework.Logf("pod %s/%s qos=%s cnt %s path %q quota %q", pod.Namespace, pod.Name, pod.Status.QOSClass, ctnName, cgPath, quota)
	return quota, nil
}

const (
	kubeCgroupRoot = "/sys/fs/cgroup"
)

// example path (systemd, crio, v2):
// /sys/fs/cgroup/ kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod0b7632a2_a56e_4278_987a_22de18008dbe.slice/ crio-conmon-0bc5eac79e3ae7a0c2651f14722aa10fa333eb2325c2ca97da33aa284cda81b0.scope
// example path (cgroup, containerd, v1):
// /sys/fs/cgroup/cpuset kubepods/burstable pod8e414e92-17c2-41de-81c7-0045bba9103b b5791f89a6971bb4a751ffbebf533399c91630aa2906d7c6b5e239f405f3b97a

func makeCgroupPathForPod(pod *v1.Pod, isV2 bool) string {
	components := []string{defaultNodeAllocatableCgroup}
	if pod.Status.QOSClass != v1.PodQOSGuaranteed {
		components = append(components, strings.ToLower(string(pod.Status.QOSClass)))
	}
	components = append(components, "pod"+string(pod.UID))

	cgroupName := cm.NewCgroupName(cm.RootCgroupName, components...)
	cgroupFsName := ""
	// it's quite ugly to use a global, but it saves us to pass a parameter all across the stack many times
	if e2enodeCgroupDriver == "systemd" {
		cgroupFsName = cgroupName.ToSystemd()
	} else {
		cgroupFsName = cgroupName.ToCgroupfs()
	}
	if !isV2 {
		cgroupFsName = filepath.Join("cpuset", cgroupFsName)
	}
	return filepath.Join(kubeCgroupRoot, cgroupFsName)
}

func makeCgroupPathForContainer(pod *v1.Pod, ctnName string, isInit, isV2 bool) (string, error) {
	fullCntID, ok := findContainerIDByName(pod, ctnName, isInit)
	if !ok {
		return "", fmt.Errorf("cannot find status for container %q", ctnName)
	}
	cntID, err := parseContainerID(fullCntID)
	if err != nil {
		return "", err
	}
	cntPath := ""
	if e2enodeCgroupDriver == "systemd" {
		cntPath = containerCgroupPathPrefixFromDriver(e2enodeRuntimeName) + "-" + cntID + ".scope"
	} else {
		cntPath = cntID
	}

	return filepath.Join(makeCgroupPathForPod(pod, isV2), cntPath), nil
}

func cpusetFileNameFromVersion(isV2 bool) string {
	if isV2 {
		return "cpuset.cpus.effective"
	}
	return "cpuset.cpus"
}

func containerCgroupPathPrefixFromDriver(runtimeName string) string {
	if runtimeName == "cri-o" {
		return "crio"
	}
	return "cri-containerd"
}

func parseContainerID(fullID string) (string, error) {
	_, cntID, found := strings.Cut(fullID, "://")
	if !found {
		return "", fmt.Errorf("unsupported containerID: %q", fullID)
	}
	// TODO: should we validate the kind?
	return cntID, nil
}

func findContainerIDByName(pod *v1.Pod, ctnName string, isInit bool) (string, bool) {
	cntStatuses := pod.Status.ContainerStatuses
	if isInit {
		cntStatuses = pod.Status.InitContainerStatuses
	}
	for idx := range cntStatuses {
		if cntStatuses[idx].Name == ctnName {
			return cntStatuses[idx].ContainerID, true
		}
	}
	return "", false
}

func makeThreadSiblingCPUSet(cpus cpuset.CPUSet) cpuset.CPUSet {
	siblingsCPUs := cpuset.New()
	for _, cpuID := range cpus.UnsortedList() {
		siblingsCPUs = siblingsCPUs.Union(cpuSiblingListFromSysFS(int64(cpuID)))
	}
	return siblingsCPUs
}

func updateKubeletConfigIfNeeded(ctx context.Context, f *framework.Framework, desiredCfg *kubeletconfig.KubeletConfiguration) *v1.Node {
	curCfg, err := getCurrentKubeletConfig(ctx)
	framework.ExpectNoError(err)

	if equalKubeletConfiguration(curCfg, desiredCfg) {
		framework.Logf("Kubelet configuration already compliant, nothing to do")
		return getLocalNode(ctx, f)
	}

	framework.Logf("Updating Kubelet configuration")
	updateKubeletConfig(ctx, f, desiredCfg, true)
	framework.Logf("Updated Kubelet configuration")

	return getLocalNode(ctx, f)
}

func equalKubeletConfiguration(cfgA, cfgB *kubeletconfig.KubeletConfiguration) bool {
	cfgA = cfgA.DeepCopy()
	cfgB = cfgB.DeepCopy()
	// we care only about the payload, force metadata to be uniform
	cfgA.TypeMeta = metav1.TypeMeta{}
	cfgB.TypeMeta = metav1.TypeMeta{}
	return reflect.DeepEqual(cfgA, cfgB)
}

type nodeCPUDetails struct {
	Capacity    int64
	Allocatable int64
	Reserved    int64
}

func cpuDetailsFromNode(node *v1.Node) nodeCPUDetails {
	localNodeCap := node.Status.Capacity
	cpuCap := localNodeCap[v1.ResourceCPU]
	localNodeAlloc := node.Status.Allocatable
	cpuAlloc := localNodeAlloc[v1.ResourceCPU]
	cpuRes := cpuCap.DeepCopy()
	cpuRes.Sub(cpuAlloc)
	// RoundUp reserved CPUs to get only integer cores.
	cpuRes.RoundUp(0)
	return nodeCPUDetails{
		Capacity:    cpuCap.Value(),
		Allocatable: cpuCap.Value() - cpuRes.Value(),
		Reserved:    cpuRes.Value(),
	}
}

// smtLevelFromSysFS returns the number of symmetrical multi-thread (SMT) execution units the processor provides.
// The most common value on x86_64 is 2 (2 virtual threads/cores per physical core), that would be smtLevel == 2.
// The following are all synonyms: threadsPerCore, smtLevel
// Note: can't find a good enough yet not overly long name, "threadSiblingCount", "smtLevel", "threadsPerCore" are all questionable.
func smtLevelFromSysFS() int {
	cpuID := int64(0) // this is just the most likely cpu to be present in a random system. No special meaning besides this.
	cpus := cpuSiblingListFromSysFS(cpuID)
	return cpus.Size()
}

func cpuSiblingListFromSysFS(cpuID int64) cpuset.CPUSet {
	data, err := os.ReadFile(fmt.Sprintf("/sys/devices/system/cpu/cpu%d/topology/thread_siblings_list", cpuID))
	framework.ExpectNoError(err)
	// how many thread sibling you have = SMT level
	// example: 2-way SMT means 2 threads sibling for each thread
	cpus, err := cpuset.Parse(strings.TrimSpace(string(data)))
	framework.ExpectNoError(err)
	return cpus
}

func uncoreCacheIDFromSysFS(cpuID int) (int64, error) {
	// expect sysfs path for Uncore Cache ID for each CPU to be:
	// /sys/devices/system/cpu/cpu#/cache/index3/id
	cacheIDPath := filepath.Join("/sys/devices/system/cpu", fmt.Sprintf("cpu%d", cpuID), "cache", "index3", "id")
	cacheIDBytes, err := os.ReadFile(cacheIDPath)
	if err != nil {
		return 0, fmt.Errorf("failed to read cache ID for CPU %d: %w", cpuID, err)
	}

	cacheIDStr := strings.TrimSpace(string(cacheIDBytes))
	cacheID, err := strconv.ParseInt(cacheIDStr, 10, 64)
	if err != nil {
		return 0, fmt.Errorf("failed to parse cache ID for CPU %d: %w", cpuID, err)
	}

	return cacheID, nil
}

func makeCPUManagerBEPod(podName string, ctnAttributes []ctnAttribute) *v1.Pod {
	var containers []v1.Container
	for _, ctnAttr := range ctnAttributes {
		ctn := v1.Container{
			Name:    ctnAttr.ctnName,
			Image:   busyboxImage,
			Command: []string{"sh", "-c", ctnAttr.ctnCommand},
			VolumeMounts: []v1.VolumeMount{
				{
					Name:      "sysfscgroup",
					MountPath: "/sysfscgroup",
				},
				{
					Name:      "podinfo",
					MountPath: "/podinfo",
				},
			},
		}
		containers = append(containers, ctn)
	}

	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers:    containers,
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
				{
					Name: "podinfo",
					VolumeSource: v1.VolumeSource{
						DownwardAPI: &v1.DownwardAPIVolumeSource{
							Items: []v1.DownwardAPIVolumeFile{
								{
									Path: "uid",
									FieldRef: &v1.ObjectFieldSelector{
										APIVersion: "v1",
										FieldPath:  "metadata.uid",
									},
								},
							},
						},
					},
				},
			},
		},
	}
}

func requireCGroupV2() {
	if e2enodeCgroupV2Enabled {
		return
	}
	e2eskipper.Skipf("Skipping since CgroupV2 not used")

}

const (
	minSMTLevel    = 2
	minCPUCapacity = 2
)

// Helper for makeCPUManagerPod().
type ctnAttribute struct {
	ctnName       string
	ctnCommand    string
	cpuRequest    string
	cpuLimit      string
	restartPolicy *v1.ContainerRestartPolicy
}

// makeCPUMangerPod returns a pod with the provided ctnAttributes.
func makeCPUManagerPod(podName string, ctnAttributes []ctnAttribute) *v1.Pod {
	var containers []v1.Container
	for _, ctnAttr := range ctnAttributes {
		cpusetCmd := "grep Cpus_allowed_list /proc/self/status | cut -f2 && sleep 1d"
		ctn := v1.Container{
			Name:  ctnAttr.ctnName,
			Image: busyboxImage,
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse(ctnAttr.cpuRequest),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
				Limits: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse(ctnAttr.cpuLimit),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			},
			Command: []string{"sh", "-c", cpusetCmd},
			VolumeMounts: []v1.VolumeMount{
				{
					Name:      "sysfscgroup",
					MountPath: "/sysfscgroup",
				},
				{
					Name:      "podinfo",
					MountPath: "/podinfo",
				},
			},
		}
		containers = append(containers, ctn)
	}

	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers:    containers,
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
				{
					Name: "podinfo",
					VolumeSource: v1.VolumeSource{
						DownwardAPI: &v1.DownwardAPIVolumeSource{
							Items: []v1.DownwardAPIVolumeFile{
								{
									Path: "uid",
									FieldRef: &v1.ObjectFieldSelector{
										APIVersion: "v1",
										FieldPath:  "metadata.uid",
									},
								},
							},
						},
					},
				},
			},
		},
	}
}

// makeCPUMangerInitContainersPod returns a pod with init containers with the
// provided ctnAttributes.
func makeCPUManagerInitContainersPod(podName string, ctnAttributes []ctnAttribute) *v1.Pod {
	var containers []v1.Container
	cpusetCmd := "grep Cpus_allowed_list /proc/self/status | cut -f2"
	cpusetAndSleepCmd := "grep Cpus_allowed_list /proc/self/status | cut -f2 && sleep 1d"
	for _, ctnAttr := range ctnAttributes {
		ctn := v1.Container{
			Name:  ctnAttr.ctnName,
			Image: busyboxImage,
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse(ctnAttr.cpuRequest),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
				Limits: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse(ctnAttr.cpuLimit),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			},
			Command:       []string{"sh", "-c", cpusetCmd},
			RestartPolicy: ctnAttr.restartPolicy,
		}
		if ctnAttr.restartPolicy != nil && *ctnAttr.restartPolicy == v1.ContainerRestartPolicyAlways {
			ctn.Command = []string{"sh", "-c", cpusetAndSleepCmd}
		}
		containers = append(containers, ctn)
	}

	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy:  v1.RestartPolicyNever,
			InitContainers: containers,
			Containers: []v1.Container{
				{
					Name:  "regular",
					Image: busyboxImage,
					Resources: v1.ResourceRequirements{
						Requests: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse("1000m"),
							v1.ResourceMemory: resource.MustParse("100Mi"),
						},
						Limits: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse("1000m"),
							v1.ResourceMemory: resource.MustParse("100Mi"),
						},
					},
					Command: []string{"sh", "-c", cpusetAndSleepCmd},
				},
			},
		},
	}
}

type cpuManagerKubeletArguments struct {
	policyName                       string
	enableCPUManagerOptions          bool
	disableCPUQuotaWithExclusiveCPUs bool
	enablePodLevelResources          bool
	reservedSystemCPUs               cpuset.CPUSet
	options                          map[string]string
}

func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, kubeletArguments *cpuManagerKubeletArguments) *kubeletconfig.KubeletConfiguration {
	newCfg := oldCfg.DeepCopy()
	if newCfg.FeatureGates == nil {
		newCfg.FeatureGates = make(map[string]bool)
	}

	newCfg.FeatureGates["CPUManagerPolicyBetaOptions"] = kubeletArguments.enableCPUManagerOptions
	newCfg.FeatureGates["CPUManagerPolicyAlphaOptions"] = kubeletArguments.enableCPUManagerOptions
	newCfg.FeatureGates["DisableCPUQuotaWithExclusiveCPUs"] = kubeletArguments.disableCPUQuotaWithExclusiveCPUs
	newCfg.FeatureGates["PodLevelResources"] = kubeletArguments.enablePodLevelResources

	newCfg.CPUManagerPolicy = kubeletArguments.policyName
	newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}

	if kubeletArguments.options != nil {
		newCfg.CPUManagerPolicyOptions = kubeletArguments.options
	}

	if kubeletArguments.reservedSystemCPUs.Size() > 0 {
		cpus := kubeletArguments.reservedSystemCPUs.String()
		framework.Logf("configureCPUManagerInKubelet: using reservedSystemCPUs=%q", cpus)
		newCfg.ReservedSystemCPUs = cpus
	} else {
		// The Kubelet panics if either kube-reserved or system-reserved is not set
		// when CPU Manager is enabled. Set cpu in kube-reserved > 0 so that
		// kubelet doesn't panic.
		if newCfg.KubeReserved == nil {
			newCfg.KubeReserved = map[string]string{}
		}

		if _, ok := newCfg.KubeReserved["cpu"]; !ok {
			newCfg.KubeReserved["cpu"] = "200m"
		}
	}

	return newCfg
}
